{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "37ab138a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import pandas as pd \n",
    "import numpy as np \n",
    "import matplotlib.pyplot as plt \n",
    "# import seaborn as sns\n",
    "import string\n",
    "import nltk\n",
    "import warnings \n",
    "from wordcloud import WordCloud\n",
    "import json\n",
    "\n",
    "warnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "b7a78710",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "If I smelled the scent of hand sanitizers today on someone in the past, I would think they were so intoxicated that… https://t.co/QZvYbrOgb0\n",
      "Hey @Yankees @YankeesPR and @MLB - wouldn't it have made more sense to have the players pay their respects to the A… https://t.co/1QvW0zgyPu\n",
      "@diane3443 @wdunlap @realDonaldTrump Trump never once claimed #COVID19 was a hoax. We all claim that this effort to… https://t.co/Jkk8vHWHb3\n",
      "@brookbanktv The one gift #COVID19 has give me is an appreciation for the simple things that were always around me… https://t.co/Z0pOAlFXcW\n",
      "25 July : Media Bulletin on Novel #CoronaVirusUpdates #COVID19 \n",
      "@kansalrohit69 @DrSyedSehrish @airnewsalerts @ANI… https://t.co/MN0EEcsJHh\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>len</th>\n",
       "      <th>cnt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>142337.000000</td>\n",
       "      <td>142337.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>130.767243</td>\n",
       "      <td>17.665245</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>19.199069</td>\n",
       "      <td>4.115441</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>13.000000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>132.000000</td>\n",
       "      <td>15.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>140.000000</td>\n",
       "      <td>18.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>140.000000</td>\n",
       "      <td>21.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>169.000000</td>\n",
       "      <td>54.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 len            cnt\n",
       "count  142337.000000  142337.000000\n",
       "mean      130.767243      17.665245\n",
       "std        19.199069       4.115441\n",
       "min        13.000000       2.000000\n",
       "25%       132.000000      15.000000\n",
       "50%       140.000000      18.000000\n",
       "75%       140.000000      21.000000\n",
       "max       169.000000      54.000000"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "covid = 'covid19_tweets.csv'\n",
    "df = pd.read_csv(covid)\n",
    "df = df[['user_name', 'user_location', 'date', 'text']]\n",
    "df = df.dropna(subset=['text', 'user_location'])\n",
    "df['len'] = df['text'].apply(lambda x: len(x))\n",
    "df['cnt'] = df['text'].apply(lambda x: len(x.split()))\n",
    "for s in df['text'][0:5]:\n",
    "    print(s)\n",
    "df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "7464f3e8",
   "metadata": {},
   "outputs": [],
   "source": [
    "#!/usr/bin/python\n",
    "# -*- coding: UTF-8 -*-\n",
    "\n",
    "import json\n",
    "import re\n",
    "import warnings\n",
    "import nltk\n",
    "import pandas as pd\n",
    "\n",
    "warnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n",
    "\n",
    "pattern_escape = re.compile(r'&\\w{2,3}')\n",
    "pattern_at = re.compile(r'@\\w+')\n",
    "pattern_marker = re.compile(r\"[^a-zA-Z#]\")\n",
    "pattern_url = re.compile('(https?|ftp|file)://[-A-Za-z0-9+&@#/%?=~_|!:,.;]+[-A-Za-z0-9+&@#/%=~_|]')\n",
    "lemma = nltk.wordnet.WordNetLemmatizer()\n",
    "\n",
    "\n",
    "def lower(s):\n",
    "    s = s.lower()  # 返回小写\n",
    "    return s\n",
    "\n",
    "\n",
    "def remove_url(s):\n",
    "    s = pattern_url.sub('', s)  ## 去除url\n",
    "    return s\n",
    "\n",
    "\n",
    "def remove_escape(s):\n",
    "    s = pattern_escape.sub('', s)\n",
    "    return s\n",
    "\n",
    "\n",
    "def remove_at(s):\n",
    "    s = pattern_at.sub('', s)  # 去除@xx\n",
    "    return s\n",
    "\n",
    "\n",
    "def remove_marker(s):\n",
    "    s = pattern_marker.sub(' ', s)  # 仅保留英文单词\n",
    "    return s\n",
    "\n",
    "def remove_short(s):\n",
    "    ss = s.split()\n",
    "    results = []\n",
    "    for x in ss:\n",
    "        if len(x) > 2:\n",
    "            results.append(x)\n",
    "    new_line = ' '.join(results)\n",
    "    return new_line\n",
    "\n",
    "def do_lemmatize(s):  # 词性还原\n",
    "    ss = s.split()\n",
    "    results = []\n",
    "    for x in ss:\n",
    "        results.append(lemma.lemmatize(x))\n",
    "    new_line = ' '.join(results)\n",
    "    return new_line\n",
    "\n",
    "\n",
    "def do_stem(s):  # 词干提取\n",
    "    ss = s.split()\n",
    "    results = []\n",
    "    for x in ss:\n",
    "        results.append(stemmer.stem(x))\n",
    "    new_line = ' '.join(results)\n",
    "    return new_line\n",
    "\n",
    "\n",
    "def clean2():\n",
    "    covid = 'covid19_tweets.csv'\n",
    "    df = pd.read_csv(covid)\n",
    "    df = df[['user_name', 'user_location', 'date', 'text']]\n",
    "    df = df.dropna(subset=['text', 'user_location'])\n",
    "    \n",
    "    df['text'] = df['text'].apply(lower)\n",
    "    df['text'] = df['text'].apply(remove_url)\n",
    "    df['text'] = df['text'].apply(remove_escape)\n",
    "    df['text'] = df['text'].apply(remove_at)\n",
    "    df['text'] = df['text'].apply(remove_marker)\n",
    "    df['text'] = df['text'].apply(remove_short)\n",
    "    df['text'] = df['text'].apply(do_lemmatize)\n",
    "#     df['text'] = df['text'].do_stem(clean_txt)\n",
    "    df['text'] = df['text'].apply(remove_short)\n",
    "    df = df.drop(df[df['text'] == ''].index)\n",
    "    df.to_csv('clean.csv')\n",
    "    return df\n",
    "\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    df = clean2()\n",
    "    df.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "bd388ae6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "smelled the scent hand sanitizers today someone the past would think they were intoxicated that\n",
      "hey and wouldn have made more sense have the player pay their respect the\n",
      "trump never once claimed #covid hoax all claim that this effort\n",
      "the one gift #covid give appreciation for the simple thing that were always around\n",
      "july medium bulletin novel #coronavirusupdates #covid\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>len</th>\n",
       "      <th>cnt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>141900.000000</td>\n",
       "      <td>141900.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>82.757787</td>\n",
       "      <td>12.528830</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>18.800706</td>\n",
       "      <td>3.197404</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>3.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>73.000000</td>\n",
       "      <td>11.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>88.000000</td>\n",
       "      <td>13.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>96.000000</td>\n",
       "      <td>15.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>130.000000</td>\n",
       "      <td>23.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 len            cnt\n",
       "count  141900.000000  141900.000000\n",
       "mean       82.757787      12.528830\n",
       "std        18.800706       3.197404\n",
       "min         3.000000       1.000000\n",
       "25%        73.000000      11.000000\n",
       "50%        88.000000      13.000000\n",
       "75%        96.000000      15.000000\n",
       "max       130.000000      23.000000"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['len'] = df['text'].apply(lambda x: len(x))\n",
    "df['cnt'] = df['text'].apply(lambda x: len(x.split()))\n",
    "for s in df['text'][0:5]:\n",
    "    print(s)\n",
    "df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "ed54006a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 141938 entries, 0 to 179107\n",
      "Data columns (total 4 columns):\n",
      " #   Column         Non-Null Count   Dtype \n",
      "---  ------         --------------   ----- \n",
      " 0   user_name      141938 non-null  object\n",
      " 1   user_location  141938 non-null  object\n",
      " 2   date           141938 non-null  object\n",
      " 3   text           141938 non-null  object\n",
      "dtypes: object(4)\n",
      "memory usage: 5.4+ MB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
